import sys
sys.path.append("public")
from demo import bot
from urllib.parse import quote
from urllib.parse import urljoin
from pyquery import PyQuery as jq
import json,re
import math
import random
import time
import logging
from openpyxl import *

keys = "手机" #关键字
mainurl = "https://jd.com"
xls_file = f"{keys}.xlsx"
xls_file2 = f"{keys}评论.xlsx"
datacol = ["商品ID","链接","价格","商品名", "评价数", "店铺","底部标签","热门标签"]
datacol2 = ["商品ID","标题","颜色","内存", "配置", "时间","评分","评论","点赞数","评论数","追评","追评时间"]
class spider(bot):
    """docstring for spider"""
    def __init__(self, mainurl=None):
        super(spider, self).__init__(mainurl)
        #翻页
        self.search_url = "https://search.jd.com/Search"#搜索链接
        #评论
        self.comments = "https://sclub.jd.com/comment/productPageComments.action?"
        self.mylist = []
        self.all_dict = {}
        self.comment_list = []
        self.wb_content_type = ""
        self.sku_id = ""
       #表格创建初始化
    def get_xls_cursor(self):
        self.wb = Workbook()
        if os.path.exists(xls_file):
            self.wb = load_workbook(xls_file)
        else:
            self.sheet = self.wb.active
            self.sheet.append(datacol)
            self.wb.save(xls_file)
        self.sheet = self.wb.active
        self.sheet.title = "datas"

        self.wb2 = Workbook()
        if os.path.exists(xls_file2):
            self.wb2 = load_workbook(xls_file2)
        else:
            self.sheet2 = self.wb2.active
            self.sheet2.append(datacol2)
            self.wb2.save(xls_file2)
        self.sheet2 = self.wb2.active
        self.sheet2.title = "datas"


        #分析他网页参数
    def data_parser(self, types, resp, jsonpcode="", shopid=None):
        try:
            if types in "page":
                jqdata = jq(resp.content)#获取网页源码
                #手机列表在 class='J_goodsList'的div下
                #解析ul li数据
                for index,item in enumerate(jqdata("#J_goodsList > ul > li").items()):
                    # if index > 2:
                        # break
                    sku = item.attr('data-sku')  #保存商品的编码
                    self.sku_id = sku
                    #获取商品名字
                    next_name = item(
                            "div > div.p-name.p-name-type-2 > a > em").text().strip().replace('\n',' ')
                    #获取价格
                    price = item(
                            "div > div.p-price > strong > i").text()
                    #获取销售量
                    comment = item(
                            "div > div.p-commit > strong > a").text()
                    #获取详情页连接
                    jsonurl = urljoin("https://item.jd.com",item(
                            "div > div.p-name.p-name-type-2 > a").attr('href'))
                    #底部标签
                    bottom_lable = item(
                            "div > div.p-icons").text()
                    #获取店铺名字
                    store_name = item("div > div.p-shop > span >  a").text()
                    self.logger.info(f"current sku : {sku}")
                    #分类的所以数据
                    hotCommentTag = self.getPercentType(sku)
                    tag_str = ""
                    for tag in hotCommentTag:
                        tag_str = tag_str + tag['name'] + "(" + str(tag['count']) + ")  "
                    self.sheet.append([
                        sku,
                        jsonurl,
                        price,
                        next_name,
                        comment,
                        store_name,
                        bottom_lable,
                        tag_str
                    ])
                    for pages in range(0,100):
                        self.getContent(pages,sku)
                    self.wb.save(xls_file)
                    self.wb2.save(xls_file2)
        except Exception as e:
            raise
    #jsonp转化为json
    def loads_jsonp(self,_jsonp):
        try:
            return json.loads(re.match(".*?({.*}).*",_jsonp,re.S).group(1))
        except:
            raise ValueError('Invalid Input')
    #解析json
    def parseJson(self,data):
        comments = data['comments']
        for comment in comments:

            if "afterUserComment" in comment:
                next_comment = comment['afterUserComment']['hAfterUserComment']['content']
                next_time = comment['afterUserComment']['created']
            else:
                next_comment = ""
                next_time = ""
            if comment['productSales']:
                neicun = comment['productSales'][0]['saleValue']
            else:
                neicun = ""
            try:
                self.sheet2.append([self.sku_id,
                    comment['referenceName'],comment['productColor'],comment['productSize'],neicun,
                    comment['referenceTime'],comment['score'],comment['content'],comment['usefulVoteCount'],
                    comment['uselessVoteCount'],
                    next_comment,
                    str(next_time)
                ]) 
            except Exception as e:
                raise



    #获取评论分类
    def getPercentType(self,skuid):
        params = {"callback":"callbackFunction",
            "productId":skuid,
            "score":"0",
            "sortType":"5",
            "page":"2",
            "pageSize":"1"}
        self.logger.info(f"current PercentType")
        resp = self.session.get(self.comments, params=params)
        if self.resp_check(resp):#请求验证
            self.logger.info("current getJson")
            myjson = self.loads_jsonp(resp.text)
            return myjson['hotCommentTagStatistics']
    #获取评论数据
    def getContent(self,page,skuid):
        params = {"callback":"callbackFunction",
            "productId":skuid,
            "score":"0",
            "fold":"1",
            "sortType":"6",
            "page":page,
            "pageSize":"10"}
        self.logger.info(f"current page_number -> [{page}]")
        try:
        	resp = self.session.get(self.comments, params=params)
	        if self.resp_check(resp):#请求验证
	            myjson = self.loads_jsonp(resp.text)
	            self.parseJson(myjson)
        except Exception as e:
        	raise
    def resp_check(self, resp, login=False):
        """响应处理，429,403错误自动切换用户"""
        if self.randomUA:
            self.crawler.random_user_agent()
        netcode = resp.status_code
        if netcode == 200:
            return True
     #获取商品列表
    def get_page(self, page):

        #get参数，用于翻页
        params = {"bs": "1",
                  "click": "0",
                  "enc": "utf-8",
                  "keyword": keys,#关键词
                  "page": page,
                  "qrst": "1",
                  "rt": "1",
                  "stop": "1",
                  "vt": "2"}
        self.logger.info(f"current page_number -> [{page}]")
        resp = self.session.get(self.search_url, params=params)
        if self.resp_check(resp):#请求验证
            self.data_parser("page", resp)
    def start(self):
        for page in range(0,1):#设置翻页数
            self.get_page(page)#翻页功能

if __name__ == '__main__':
    try:
        myspider = spider("https://jd.com")
        myspider.get_xls_cursor()
        myspider.start()
    except KeyboardInterrupt:
        pass
